/*
 * practice code while learning <lucene in action>
 * indexer
 * mengchang@taobao.com
 * 2011-5-26
 */
package lucenetest;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Version;

public class TestIndexer 
{
	public static void main(String[] args)
	{
		System.out.println("in TestIndexer");
		//这个是lucene的javadoc文档所在的目录，其中的html文件，用英语写成
		String sourcedir="F:\\learning\\lucene-3.1.0\\lucene-3.1.0\\docs";
		String indexdir="F:\\learning\\lucenIndex";
		if (System.getProperty("os.name").equalsIgnoreCase("Linux")) 
		{
			sourcedir="/home/jiening/learning/lucene-3.1.0/lucene-3.1.0/docs";
			indexdir="/home/jiening/learning/luceneTestIndex";
		}
		System.out.println("sourcedir="+sourcedir);
		System.out.println("indexdir="+indexdir);
		
		// 扫描要处理的文件（sourcedir中以html为后缀的），文件列表保存在doclist中
		docid = 1L;
		doclist.clear();
		scanHtmlInDir(sourcedir, "html");
		
		// 构造神器
		Directory idxdir;
		IndexWriter writer = null;
		try 
		{
			idxdir = new SimpleFSDirectory(new File(indexdir));
			IndexWriterConfig iwconf = new IndexWriterConfig(Version.LUCENE_31, new StandardAnalyzer(Version.LUCENE_31));
			writer = new IndexWriter(idxdir, iwconf);
		} 
		catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return;
		}
		
		// 逐个索引保存在doclist中的文件
		Document doc = null;
		for (Map.Entry<Long, String> thedocfile : doclist.entrySet())
		{
			System.out.println(thedocfile.getKey()+" -> "+thedocfile.getValue());
			doc = new Document();
			try {
				doc.add(new Field("docid", thedocfile.getKey().toString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("path", thedocfile.getValue(), Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("content", new FileReader(thedocfile.getValue()), Field.TermVector.WITH_POSITIONS_OFFSETS));
				writer.addDocument(doc);
			} catch (FileNotFoundException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();				
			} catch (CorruptIndexException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			//break;
		}
		
		// 索引优化，提交持久化，神器关闭
		try {
			writer.optimize();
			writer.commit();
			writer.close();
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		System.out.println("docid="+docid);
	}
	
	private static long docid=0L;
	private static HashMap<Long, String> doclist = new HashMap<Long, String>();
	
	// 递归扫描，将后缀为subfix的文件放到doclist中
	private static void scanHtmlInDir(String dirpath, String subfix)
	{
		File nowdir = new File(dirpath);
		if(!nowdir.isDirectory())
		{
			System.out.println(dirpath+" is not a dir");
			return;
		}
		File[] filelist = nowdir.listFiles();
		if(null == filelist || 0 == filelist.length)
		{
			return;
		}

		for(File nowfile : filelist)
		{
			//System.out.println(nowfile.getAbsolutePath());
			if(nowfile.isDirectory())
			{
				// 找到目录向下递归
				scanHtmlInDir(nowfile.getAbsolutePath(), subfix);
			}
			else
			{
				if(0 == subfix.length())
				{
					System.out.println(docid+" -> "+nowfile.getAbsolutePath());
					doclist.put(docid, nowfile.getAbsolutePath());
					++docid;
				}
				else
				{
					if(nowfile.getName().endsWith(subfix))
					{
						//System.out.println(docid+" -> "+nowfile.getAbsolutePath());
						doclist.put(docid, nowfile.getAbsolutePath());
						++docid;
					}
				}//end of if(0 == subfix.length())
			}// end of if(nowfile.isDirectory())
		}// end of for		
	}
}